package othello.players.learnplayers;

import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.Random;
import java.awt.Color;

import othello.model.Action;
import othello.model.IBoard;
import othello.model.State;
import othello.neuralnetwork.NeuralNetwork;

public class PosTD extends AbstractNeuralNetworkPlayer {
    private static final double epsInit = 0.1;
    private static final double tBase = 0.9999995;    // got the t-values from a paper
    private static final double tThreshold = 0.002;	 
    private static final double lambda_decay = 0.75;
    private final boolean softmax = false;
    private final boolean epsgreedy = true;
    private List<State> previousStates = new ArrayList<State>();    
    private State lastState;
    private int totalGames = -1;
    
    private static final int[][] positionValues = new int[][]{{100, -20, 10,  5,  5, 10, -20, 100},
                                                 {-20, -50, -2, -2, -2, -2, -50, -20},
                                                 { 10,  -2, -1, -1, -1, -1,  -2,  10},
                                                 {  5,  -2, -1, -1, -1, -1,  -2,   5},
                                                 {  5,  -2, -1, -1, -1, -1,  -2,   5},
                                                 { 10,  -2, -1, -1, -1, -1,  -2,  10},
                                                 {-20, -50, -2, -2, -2, -2, -50, -20},
                                                 {100, -20, 10,  5,  5, 10, -20, 100}};
    
    public PosTD(NeuralNetwork neuralNetwork) {
        super("PosTD", neuralNetwork);
    }
    
    @Override
    public void beginTurn() {
    	if (totalGames == -1){
    		totalGames = board.getRealMaxGames();
    	}
    	
    	//select which action to take according to some policy
    	
    	//Action action = selectActionPos;
        Action action = selectActionPos();
        IBoard hypoBoard = board;
        
        if (action != null)        
        	hypoBoard = board.hypotheticallyPlacePiece(this.color, action.getRow(), action.getCol());
                
    	State state = new State(cloneByteArray(hypoBoard.getState()));
        
        // update NN        
        if (this.benchmarkState == BenchmarkState.Training && lastState != null){        	
        	nn.forwardPropagate(state.toDoubles());
        	double newV = nn.getOutput()[0];
        	nn.forwardPropagate(lastState.toDoubles());
        	double oldV = nn.getOutput()[0];
        	double error = newV - oldV;
        	
        	if(useEligibility){
        		previousStates.add(state);
        		double lambda = 1;
                
                ListIterator<State> li = previousStates.listIterator(previousStates.size());   
                State stateItr = li.previous();
                
                while(li.hasPrevious()) {
                	stateItr = li.previous();
                    nn.forwardPropagate(stateItr.toDoubles());
                    double decayedError = lambda*error;
                    nn.backPropagate(decayedError, nnLR, 0);
                    lambda *= lambda_decay;    	        	
                }        		
        		
        	} else {
        		nn.backPropagate(error, nnLR, 0);
        	}
        }               

        if (action == null){
        	board.pass(this);
        } else {            
            lastState = state;
        	board.placePiece(this, action.getRow(), action.getCol());
        }
    }
    
	@Override
    public void endGame() {
		// Update NN
    	if (this.benchmarkState == BenchmarkState.Training && lastState != null) {
    		nn.forwardPropagate(lastState.toDoubles());
        	double oldV = nn.getOutput()[0];
        	double reward = reward(board);        	
        	double error = reward - oldV;
        	//System.out.print("Reward " + getColor() + ": " + reward + ", oldV: " + oldV + '\n');
        	
        	
        	if(useEligibility){
        		double lambda = 1;                
                ListIterator<State> li = previousStates.listIterator(previousStates.size());                                
                while(li.hasPrevious()) {
                    State stateItr = li.previous();
                    nn.forwardPropagate(stateItr.toDoubles());
                    double decayedError = lambda*error;
                    nn.backPropagate(decayedError, nnLR, 0);
                    lambda *= lambda_decay;    	        	
                }        		
        		
        	} else {
        		nn.backPropagate(error, nnLR, 0);
        	}
        	
        	lastState = null;
        	this.previousStates.clear();
        	
    	}
    }
        
    private Action selectAction(){
        List<Action> availableActions = getAvailableActions(board);
        Action bestAction = null;
        Random generator = new Random();
        
        if (availableActions.isEmpty()){
            return null;
        }
        
        if (this.benchmarkState == BenchmarkState.Benchmarking || this.benchmarkState == BenchmarkState.Playing ) {
        	if (board.getTurns() < 8 && board.getRandFirstFourMoves()){
    			bestAction = availableActions.get(generator.nextInt(availableActions.size()));
            } else {
                bestAction = getMaxMove(availableActions);
            }
        } else if(epsgreedy){
        	bestAction = getEpsGreedyMove(availableActions);
        } else if (softmax) {
        	bestAction = getSoftMaxMove(availableActions);
        }
        
        return bestAction;
	}
	
	private Action selectActionPos()
	{		
		if (board.getTurns() < 8 && (board.getBenchmarkstate() == BenchmarkState.Benchmarking && board.getRandFirstFourMoves()))
    		return selectRandomAction();		
        else {                			
			int maxEval = Integer.MIN_VALUE;
			int bestR = -1;
			int bestC = -1;
			int eval;
			boolean inEndGame = checkEndGame(board);
			
					
			
			for (int r = 0; r < board.getRowCount(); r++) {
				for (int c = 0; c < board.getColCount(); c++) {
					if (board.isValidMove(getColor(), r, c)) {    
						IBoard hypoBrd = board.hypotheticallyPlacePiece(this.getColor(), r, c);
						if (inEndGame){
							eval = evalEndgame(hypoBrd);
						} else {
							eval = evalWeighted(hypoBrd);
						}
						if (eval > maxEval){
							maxEval = eval;
							bestR = r;
							bestC = c;
						}
					}
				}         
			}
			
			if (bestR == -1){
				return null;
			}
			
			return new Action(bestR,bestC);
		}
	}
	
	private Action selectRandomAction() {
		List<Action> availableActions = getAvailableActions(board);
		Action bestAction = null;
		Random generator = new Random();
		
		if (!availableActions.isEmpty())
			bestAction = availableActions.get(generator.nextInt(availableActions.size()));		
		
		return bestAction;	
    }
	
	private int evalWeighted(IBoard brd) {
        int eval = 0;
        for (int r = 0; r < brd.getRowCount(); r++) {
            for (int c = 0; c < brd.getColCount(); c++) {
                eval += positionValues[r][c]*positionOccupance(brd,r,c);
            }
        }
        return eval;
    }
    
    private int evalEndgame(IBoard brd) {
        int eval = 0;
        for (int r = 0; r < brd.getRowCount(); r++) {
            for (int c = 0; c < brd.getColCount(); c++) {
                eval += positionOccupance(brd,r,c);
            }
        }
        return eval;
    }
    
    private int positionOccupance(IBoard brd, int r, int c) {
        Color color = brd.playerAt(r,c);
        if (color == null){
            return 0;
        }
        if (color.equals(this.getColor())) {
            return 1;
        }
        
        return -1;
    }
    
    private double percentOccupied(IBoard brd){
        double count = 0.0;
        for (int r = 0; r < brd.getRowCount(); r++) {
            for (int c = 0; c < brd.getColCount(); c++) {
                count += positionOccupance(brd,r,c)*positionOccupance(brd,r,c);
            }
        }                    
        return (count/64d)*100d;
    }
    
    private boolean checkEndGame(IBoard brd) {
        int cornersCovered  =   positionOccupance(brd,0,0)*positionOccupance(brd,0,0) +
                                positionOccupance(brd,0,7)*positionOccupance(brd,0,7) +
                                positionOccupance(brd,7,0)*positionOccupance(brd,7,0) +
                                positionOccupance(brd,7,7)*positionOccupance(brd,7,7);
                                
        return (cornersCovered == 4) || (percentOccupied(brd) >= 80.0) ;
    }
    
    public Action getMaxMove(List<Action> availableActions) {
    	Action bestAction = null;
        double max = -1.0 * Double.MAX_VALUE;
        double value;        

        for (Action a : availableActions){        	
        	IBoard hypoBoard = board.hypotheticallyPlacePiece(this.color, a.getRow(), a.getCol());
        	State newState = new State(cloneByteArray(hypoBoard.getState()));
        	nn.forwardPropagate(newState.toDoubles());
        	double output[] = nn.getOutput();
        	value = output[0];
            
        	if (value > max){
                max = value;
                bestAction = a;
            }
        }
        
        if (bestAction != null){
        	bestAction.setQValue(max);
        }       
        return bestAction;
    }

    public Action getEpsGreedyMove(List<Action> availableActions) {
    	Random generator = new Random();
    	Action returnAction = null;
    	double eps = epsInit*(double)(board.getGamesLeft())/(double)totalGames;
          
        if (generator.nextDouble() < eps){
            returnAction = availableActions.get(generator.nextInt(availableActions.size()));
            IBoard hypoBoard = board.hypotheticallyPlacePiece(this.color, returnAction.getRow(), returnAction.getCol());
            State newState = new State(cloneByteArray(hypoBoard.getState()));
            nn.forwardPropagate(newState.toDoubles());
            double output[] = nn.getOutput();            
            returnAction.setQValue(output[0]);                    
        } else {
            double max = -1.0 * Double.MAX_VALUE;
            double value;
            Action bestAction = null;

            for (Action a : availableActions) {
                    IBoard hypoBoard = board.hypotheticallyPlacePiece(this.color, a.getRow(), a.getCol());
                    State newState = new State(cloneByteArray(hypoBoard.getState()));
                    nn.forwardPropagate(newState.toDoubles());
                    double output[] = nn.getOutput();
                    value = output[0];

                    if (value > max){
                    max = value;
                    bestAction = a;
                }
            }

            if (bestAction != null){
                    bestAction.setQValue(max);
            }
            
            returnAction = bestAction;
        }
        
        return returnAction;
    }
    
    public Action getSoftMaxMove(List<Action> availableActions) {
    	Random generator = new Random();
    	Action bestAction = null;
    	double quotient = 0.0;
    	double gamesPlayed = (totalGames - board.getGamesLeft())*15000000.0/totalGames;
    	double T = Math.pow(tBase,gamesPlayed);
    	double[] stateValues = new double[availableActions.size()];
    	int actionIdx = 0;    	
    	int bestIdx = 0;
    	
    	for (Action a : availableActions){
    		IBoard hypoBoard = board.hypotheticallyPlacePiece(this.color, a.getRow(), a.getCol());
        	State newState = new State(cloneByteArray(hypoBoard.getState()));
        	nn.forwardPropagate(newState.toDoubles());
        	double output[] = nn.getOutput();
        	stateValues[actionIdx++] = output[0];
        	quotient += Math.exp(output[0]/T);
    	}    	
    	if (T < tThreshold){	//Just take max when T is below threshold
	        double max = -1.0 * Double.MAX_VALUE;
	        double value;
	
	        actionIdx = 0;
	        for (Action a : availableActions){		        	
	        	value = stateValues[actionIdx++];	            
	        	if (value > max){
	                max = value;
	                bestAction = a;
                    bestAction.setQValue(max);
	            }
	        }
    		
    	} else {
    		double rnd = generator.nextDouble();
        	double probA = 0.0;        	
        	boolean first = true;
        	actionIdx = 0;
        	for (Action a : availableActions){
        		probA += Math.exp(stateValues[actionIdx++]/T)/quotient;
        		if (probA >= rnd && first){
        			bestAction = a;
        			bestIdx = actionIdx - 1;        			
                    bestAction.setQValue(stateValues[bestIdx]);
        			first = false;
        		}
        	}
    	}   	
    	return bestAction;
    }

    @Override
    public void endTurn() {
    }
}
